 /**
 * 上海中赢金融信息服务有限公司
 * Copyright (c) 2017-2027 Chinazyjr,Inc.All Rights Reserved.
 */
package com.sys.midware.httpClient.crawler;

import java.io.IOException;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;

import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.client.ClientProtocolException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class Crawler {

    private static final Logger logger = LoggerFactory.getLogger("crawler");
    public static List<HttpClient> clientList = new LinkedList<HttpClient>();
    
    private static ThreadLocal<HttpClient> CLOCK = new ThreadLocal<HttpClient>();

    private Options options = new Options();

    /**
     * 返回一个新的Crawler实例
     * 
     * @return
     */
    public static Crawler get() {
        Crawler crawler = new Crawler();
//        crawler.proxy(true);
//        crawler.proxyAddress("172.16.1.99");
//        crawler.proxyPort(4321);
        return crawler;
    }

    /**
     * 设置要爬取的URL
     * 
     * @param url
     * @return
     */
    public Crawler crawl(String url) {
        getOptions().setUrl(url);
        return this;
    }

    /**
     * 设置超时时间<br>
     * 默认是5000毫秒
     * 
     * @param timeOutInMills
     * @return
     */
    public Crawler timeOut(int timeOutInMills) {
        getOptions().setTimeOut(timeOutInMills);
        return this;
    }

    /**
     * 设置页面所用的字符集<br>
     * 默认使用header或者Meta中的字符集
     * 
     * @param charset
     * @return
     */
    public Crawler charset(String charset) {
        getOptions().setCharset(charset);
        return this;
    }

    /**
     * 当IO异常发生时，重试多少次<br>
     * 默认不重试
     * 
     * @param maxCount
     * @return
     */
    public Crawler retry(int maxCount) {
        getOptions().setRetry(maxCount);
        return this;
    }

    /**
     * 设置http请求的方法<br>
     * 默认是get请求
     * 
     * @param method
     * @return
     */
    public Crawler method(String method) {
        HttpMethod hMethod = HttpMethod.codeOf(method);
        if (hMethod == null) {
            throw new IllegalArgumentException("illegal method " + method);
        }
        getOptions().setMethod(HttpMethod.codeOf(method));
        return this;
    }
    
    public Crawler proxy(boolean useProxy){
        getOptions().setUseProxy(useProxy);
        return this;
    }
    
    public Crawler proxyAddress(String proxyAddr){
        getOptions().setProxyAddr(proxyAddr);
        return this;
    }
    
    public Crawler proxyPort(int proxyPort){
        getOptions().setProxyPort(proxyPort);
        return this;
    }

    /**
     * 设置请求的header
     * 
     * @param name
     * @param value
     * @return
     */
    public Crawler header(String name, Object value) {
        getOptions().getHeaders().put(name, value);
        return this;
    }

    /**
     * 设置请求的header
     * 
     * @param headers
     * @return
     */
    public Crawler header(Map<String, Object> headers) {
        getOptions().getHeaders().putAll(headers);
        return this;
    }

    /**
     * 设置请求的payload,仅支持post请求
     * 
     * @param payload
     * @return
     */
    public Crawler payload(Map<String, Object> payload) {
        getOptions().getPayload().putAll(payload);
        return this;
    }

    /**
     * 
     * 设置请求的payload,仅支持post请求
     * 
     * @param name
     * @param value
     * @return
     */
    public Crawler payload(String name, Object value) {
        getOptions().getPayload().put(name, value);
        return this;
    }

    /**
     * 设置请求的payload,仅支持post请求
     * 
     * @param payload
     * @return
     */
    public Crawler payload(String payload) {
        getOptions().getPayload().put(payload, null);
        return this;
    }

    /**
     * 设置请求的payload,仅支持post请求
     * 
     * @param payloadCharset
     * @return
     */
    public Crawler payloadCharset(String payloadCharset) {
        getOptions().setPayloadCharset(payloadCharset);
        return this;
    }

    /**
     * 
     * 爬取结果以String类型返回<br>
     * 可能是html，json等
     * 
     * @return
     * @throws ClientProtocolException
     * @throws IOException
     * @throws HttpException
     *             当status code不是200时
     */
    public String resultAsString() throws ClientProtocolException,
            HttpException, IOException {
        HttpResult result = getResult();
        checkHttpStatus(result);
        if (getOptions().getCharset() != null) {
            result.setCharset(getOptions().getCharset());
        }
        return result.getContentAsString();
    }

    /**
     * 爬取结果byte[]类型返回<br>
     * 可能是html,json,图片等
     * 
     * @return
     * @throws ClientProtocolException
     * @throws IOException
     * @throws HttpException
     *             当status code不是200时
     */
    public byte[] resultAsByte() throws ClientProtocolException, HttpException,
            IOException {
        HttpResult result = getResult();
        checkHttpStatus(result);
        return result.getContent();
    }

    /**
     * 爬取结果以HttpResult类型返回<br>
     * 包含所有response信息
     * 
     * @return
     * @throws ClientProtocolException
     * @throws IOException
     */
    public HttpResult result() throws ClientProtocolException, IOException {
        return getResult();
    }

    private void checkHttpStatus(HttpResult result) throws HttpException {
        Options options = getOptions();
        String url = options.getUrl();

        if (result == null) {
            throw new HttpException("http got null from url " + url);
        }

        int status = result.getStatus();
        if (status != 200) {
            HttpException e = new HttpException("http error,status " + status
                    + ",url " + url);
            e.setStatus(status);
            throw e;
        }
    }

    private HttpResult getResult() throws ClientProtocolException,
            HttpException, IOException {
        Options options = getOptions();
        HttpMethod method = options.getMethod();
        String url = options.getUrl();
        int retry = options.getRetry();
        int timeoutMills = options.getTimeOut();

        if (StringUtils.isBlank(url)) {
            throw new IllegalArgumentException("invalid url " + url);
        }
        
        boolean useProxy = options.isUseProxy();
        String proxyAddr = options.getProxyAddr();
        int proxyPort = options.getProxyPort();

        if (StringUtils.isBlank(url)) {
            throw new IllegalArgumentException("invalid url " + url);
        }
        if(useProxy && StringUtils.isBlank(proxyAddr) ){
            throw new IllegalArgumentException("proxy address:"+ proxyAddr+ " is invalid.");
        }
        if(useProxy && proxyPort <= 0 ){
            throw new IllegalArgumentException("proxy port:"+proxyPort+ " is invalid.");
        }

        // support http only ...
        if (!StringUtils.startsWithIgnoreCase(url, "http")) {
            url = "http://" + url;
        }

        if (retry <= 0) {
            retry = 1;
        }

        if (timeoutMills <= 0) {
            timeoutMills = 5000;
        }

        HttpResult result = null;
        for (int count = 0; count < retry; count++) {
            try {
                HttpClient client = Holder.http();
                if(useProxy){
                    client.setProxy(proxyAddr, proxyPort);
                }
                if(MapUtils.isNotEmpty(options.getHeaders())){
                    client.setHeaders(options.getHeaders());
                }
                
                if(MapUtils.isNotEmpty(getOptions().getPayload())){
                    client.setParams(getOptions().getPayload());
                }
                
                if(StringUtils.isNotBlank(getOptions().getCharset())){
                    client.setCharset(getOptions().getCharset());
                }
                
                if (HttpMethod.GET.equals(method)) {
                        result = client.get(url, timeoutMills);
                } else if (HttpMethod.HEAD.equals(method)) {
                    result = client.head(url, timeoutMills);
                } else if (HttpMethod.POST.equals(method)) {
                    String payloadCharset = options.getPayloadCharset();
                    if(StringUtils.isNotBlank(payloadCharset)){
                        client.setCharset(payloadCharset);
                    }
                    result = client.post(url,timeoutMills);
                } else {
                    throw new IllegalArgumentException("not support method "
                            + method);
                }
                return result;
            } catch (ClientProtocolException e) {
                throw e;
                // needs to throw HttpException,we can't catch all
                // exception
            } catch (IOException e) {
                if (count >= retry) {
                    throw e;
                } else {
                    logger.error("{} got error,retrying {}", url, count, e);
                }
            }
        }

        return null;
    }

    private Options getOptions() {
        return options;
    }

    private static class Options {

        private String url;

        private int retry = 1;

        private int timeOut = 5000;

        private HttpMethod method = HttpMethod.GET;

        private String charset;

        private String payloadCharset;

        private Map<String, Object> headers = new LinkedHashMap<String, Object>();

        private Map<String, Object> payload = new LinkedHashMap<String, Object>();
        
        private boolean useProxy = false;
        
        private String proxyAddr;
        
        private int proxyPort;

        public String getCharset() {
            return charset;
        }

        public void setCharset(String charset) {
            this.charset = charset;
        }

        public String getUrl() {
            return url;
        }

        public void setUrl(String url) {
            this.url = url;
        }

        public int getRetry() {
            return retry;
        }

        public void setRetry(int retry) {
            this.retry = retry;
        }

        public HttpMethod getMethod() {
            return method;
        }

        public void setMethod(HttpMethod method) {
            this.method = method;
        }

        public int getTimeOut() {
            return timeOut;
        }

        public void setTimeOut(int timeOutMills) {
            this.timeOut = timeOutMills;
        }

        public Map<String, Object> getHeaders() {
            return headers;
        }

        public Map<String, Object> getPayload() {
            return payload;
        }

        public String getPayloadCharset() {
            return payloadCharset;
        }

        public void setPayloadCharset(String payloadCharset) {
            this.payloadCharset = payloadCharset;
        }

        public boolean isUseProxy() {
            return useProxy;
        }

        public void setUseProxy(boolean useProxy) {
            this.useProxy = useProxy;
        }

        public String getProxyAddr() {
            return proxyAddr;
        }

        public void setProxyAddr(String proxyAddr) {
            this.proxyAddr = proxyAddr;
        }

        public int getProxyPort() {
            return proxyPort;
        }

        public void setProxyPort(int proxyPort) {
            this.proxyPort = proxyPort;
        }
        

    }

    private static class Holder {
        //public static final HttpClient http = new HttpClient();
        public static HttpClient http(){
            HttpClient client = CLOCK.get();
            Thread t = Thread.currentThread();
            if(client != null){
                return client;
            }else{
//              httplog.info("000000000000" + t.getName() + ",size:" + clientList.size());
//              System.out.println("000000000000" + t.getName() + ",size:" + clientList.size());
                if(clientList.size() > 100){
//                  System.out.println("11111111111"+ t.getName() + ",size:"+clientList.size());
//                  httplog.info("111111111111"+ t.getName() + ",size:"+clientList.size());
                    int mathCount = (int)(Math.random()*100);
//                  httplog.info(">>>>Math random:"+ mathCount);
                    //随机获取100以内的数字
                    return clientList.get(mathCount);
                }else{
                    client = new HttpClient();
                    CLOCK.set(client);
                    clientList.add(client);
//                  httplog.info("222222222222"+ t.getName() + ",size:"+clientList.size());
//                  System.out.println("222222222222"+ t.getName() + ",size:"+clientList.size());
                }
                
            }
            return client;
        }
    }
    
    public static void main(String[] args) throws ClientProtocolException,
            IOException, HttpException {
        
//      System.out.println(Crawler.get().timeOut(10000)
//              .crawl("http://www.meilishuo.com/guang/catalog/bag?nid=221543&cata_id=5000000000000&pstrc=fe_pos%3Awlc_navwords_0_0")
//              .method("get")
//              .header("Content-Type", "application/x-www-form-urlencoded")
//              .payload(payload).resultAsString());
    }


}
